#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: send_test_data_to_beanstalk.py
@time: 2018/1/10 10:38
"""

import json
import sys

sys.path.append('..')
sys.path.append('../..')
from common.pybeanstalk import PyBeanstalk
from config.beanstalk_conf import BEANSTALK_CONF

from common.mongo import MongDb
from config.mongo_conf import MONGO_DB_TARGET, MONGO_DB_SOURCE

# 日志模块
from logger import Logger

log = Logger('send_to_beanstalk.log').get_logger()

source_db = MongDb(MONGO_DB_SOURCE['host'], MONGO_DB_SOURCE['port'], MONGO_DB_SOURCE['db'],
                   MONGO_DB_SOURCE['username'],
                   MONGO_DB_SOURCE['password'], log=log)
target_db = MongDb(MONGO_DB_TARGET['host'], MONGO_DB_TARGET['port'], MONGO_DB_TARGET['db'],
                   MONGO_DB_TARGET['username'],
                   MONGO_DB_TARGET['password'], log=log)

APP_DATA_DB_SOURCE = {
    'host': '172.16.215.16',
    'port': 40042,
    'db': 'app_data',
    'username': 'work',
    'password': 'haizhi'
}
app_data_db = MongDb(APP_DATA_DB_SOURCE['host'], APP_DATA_DB_SOURCE['port'], APP_DATA_DB_SOURCE['db'],
                     APP_DATA_DB_SOURCE['username'],
                     APP_DATA_DB_SOURCE['password'], log=log)

# 初始化监听消息队列
beanstalk = PyBeanstalk(BEANSTALK_CONF['host'], BEANSTALK_CONF['port'])


def main():
    seed_table = 'new_gansu_company_list'
    output_tube = 'gsxt_detail_parse'
    table_name = 'new_gansu_detail_page'
    app_data_table_name = 'enterprise_data_gov'
    with open('gansu.txt', 'w') as p_file:
        for item in source_db.traverse_batch_field(seed_table, {'crawl_status': 1}, ['_id']):
            company = item.get('_id')

            result_item = target_db.find_one(table_name, {'_id': company})
            if result_item is not None:
                send_result = beanstalk.put(output_tube, json.dumps(result_item))
                if send_result is None:
                    log.error("数据太大: company = {} ".format(company))
                    continue

                if app_data_db.find_one(app_data_table_name, {'company': company}) is not None:
                    p_file.write(company + '\n')

                log.info("数据发送成功: company = {} ".format(company))

    # for item in target_db.traverse_batch(table_name, {}):
    #     if '__change_info' not in item:
    #         continue
    #     _id = item.get('_id')
    #     result_item = beanstalk.put(output_tube, json.dumps(item))
    #     if result_item is None:
    #         log.error("数据太大: company = {} ".format(_id))
    #         continue
    #
    #     log.info("数据发送成功: company = {} ".format(_id))

    log.info("数据发送完成...")


if __name__ == '__main__':
    main()
